<!doctype html>



  


<html class="theme-next mist use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.0" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="Python, 爬虫, 数据挖掘, 机器学习, 自动化测试框架" />





  <link rel="alternate" href="/atom.xml" title="Kingname" type="application/atom+xml" />




  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.0" />






<meta name="description" content="爬虫工程师，极客学院讲师，Python, Scrapy, MongoDB, Redis, Pandas, Golang。">
<meta property="og:type" content="website">
<meta property="og:title" content="Kingname">
<meta property="og:url" content="https://kingname.info/page/3/index.html">
<meta property="og:site_name" content="Kingname">
<meta property="og:description" content="爬虫工程师，极客学院讲师，Python, Scrapy, MongoDB, Redis, Pandas, Golang。">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Kingname">
<meta name="twitter:description" content="爬虫工程师，极客学院讲师，Python, Scrapy, MongoDB, Redis, Pandas, Golang。">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Mist',
    sidebar: {"position":"left","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="https://kingname.info/page/3/"/>





  <title> Kingname </title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  










  
  
    
  

  <div class="container one-collumn sidebar-position-left 
   page-home 
 ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">Kingname</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle">给时光以生命。</p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tweet">
          <a href="/tweet" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-twitter"></i> <br />
            
            时间
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
  <section id="posts" class="posts-expand">
    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2016/05/19/screenshot/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/05/19/screenshot/" itemprop="url">
                  使用Python保存截图
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2016-05-19T22:47:29+08:00">
                2016-05-19
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2016/05/19/screenshot/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2016/05/19/screenshot/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <h2 id="起因"><a href="#起因" class="headerlink" title="起因"></a>起因</h2><p>在<a href="http://jikexueyuan.com" target="_blank" rel="external">极客学院</a>讲授《使用Python编写远程控制程序》的课程中，涉及到查看被控制电脑屏幕截图的功能。</p>
<p>如果使用PIL，这个需求只需要三行代码：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> ImageGrab</div><div class="line">pic = ImageGrab.grab()</div><div class="line">pic.save(<span class="string">'1.jpg'</span>)</div></pre></td></tr></table></figure></p>
<p>但是考虑到被控端应该尽量的精简，对其他模块尽量少的依赖，这样才能比较方便的部署，因此我考虑能否有一种方法，不依赖PIL来实现截图的功能。</p>
<h2 id="思路"><a href="#思路" class="headerlink" title="思路"></a>思路</h2><p>由于被控端使用了win32api, 因此有一个方法：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">win32api.keybd_event</div></pre></td></tr></table></figure></p>
<p>这个方法可以模拟键盘的按键动作。因此，解决方法就比较的明显了：</p>
<ol>
<li>模拟键盘上面的“Print Screen” 键按下</li>
<li>从剪贴板中读取出截图</li>
<li>将截图保存到本地 </li>
</ol>
<p>第一步非常的简单，实用win32api 和 win32con，两行代码就能实现：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">import win32api</div><div class="line">import win32con</div><div class="line">win32api.keybd_event(win32con.VK_SNAPSHOT, 0)</div></pre></td></tr></table></figure></p>
<p>其中win32con这个库里面包含了很多定义好的和Windows相关的常量，而VK_SNAPSHOT就是Print Screen键的键位码。后面的数字0表示截取整个屏幕。如果改成数字1，表示截取当前窗口。</p>
<p>那么现在问题来了，在不实用PIL的情况下，如何将剪贴板你们的图片保存到本地？</p>
<p>win32api有一个模块 win32clipboard 是负责剪贴板相关的操作。它有一个方法：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">win32clipboard.GetClipboardData(formats)</div></pre></td></tr></table></figure></p>
<p>这个方法可以从剪贴板里面读取数据。但是需要指定数据的格式。从<a href="https://msdn.microsoft.com/en-us/library/windows/desktop/ff729168\(v=vs.85\" target="_blank" rel="external">这里</a>.aspx)可以查看到更多的标准剪贴板格式(<em>Standard Clipboard Formats</em>).</p>
<p>一开始我使用的formats是<strong>CF_BITMAP</strong>，程序返回的是一串整数，怀疑应该是一个内存地址。这也和这个format的描述：</p>
<blockquote>
<p>A handle to a bitmap (<strong>HBITMAP</strong>).</p>
</blockquote>
<p>是一致的，它是一个handle。</p>
<p>我也尝试过CF_TIFF, 不过程序直接报错了，可见我使用Print Screen截图以后，剪贴板里面的图片格式并不是TIFF。</p>
<p>经过查阅其他资料，我最后确定使用了CF_DIB。</p>
<blockquote>
<p>A memory object containing a BITMAPINFO structure followed by the bitmap bits.</p>
</blockquote>
<p>这个描述说明，CF_DIB返回的是一个内存对象，包含了BIT格式图片的信息。经过测试使用：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">win32clipboard.GetClipboardData(win32con.CF_DIB)</div></pre></td></tr></table></figure></p>
<p>以后，可以得到一个很大的字符串。显然这个字符串就是图片的内容了。但是当我把这个字符串写入到bmp格式的文件后，却发现图片无法打开。</p>
<h2 id="解决办法"><a href="#解决办法" class="headerlink" title="解决办法"></a>解决办法</h2><p>在StackOverflow上，我遇到了一个非常好的老先生： <a href="http://stackoverflow.com/users/355230/martineau" target="_blank" rel="external">Mr. martineau</a>他为了解答了问题，并给我提供了解决办法。以下内容翻译自martineau先生的回答，原文请戳-&gt;<a href="http://stackoverflow.com/a/35885108/3922976" target="_blank" rel="external">http://stackoverflow.com/a/35885108/3922976</a></p>
<blockquote>
<p>你的方法的主要问题在于，你写入文件的字符串缺少了.bmp 文件头，这个文件头是<code>BITMAPFILEHEADER</code>结构。</p>
<p>为了创建这个文件头，使用<code>GetClipboardData()</code>返回的字符串必须要进行解码(<em>decoded</em>)。对于<code>CF_DIB</code>格式来说，返回的字符串的前面一部分就是<code>BOTMAPINFOHEADER</code>。</p>
<p>对于各种各样有不同种类压缩的<code>DIB</code>来说，这种文件头结构是非常的普遍的。不过幸好对截图来说，只需要简单的无压缩的RGBA像素。</p>
<p>由于<code>BOTMAPFILEHEADER</code>被放在了bf0ffBits的区域里，所以事情就变得很容易了。而其他的情况，例如大尺度的颜色表跟在<code>BITMAPINFOHEADER</code> 和像素数组的开头。</p>
<p>（这一段我看不太懂，还请如果有能正确解释这段话的朋友指正。原文是：</p>
<blockquote>
<p>That fact makes things much easier because otherwise determining the value to put in the bfOffBits field of the BITMAPFILEHEADER would be complicated by the fact that in most other cases there’s also a variably-sized color table following the BITMAPINFOHEADER and the start of the pixel array.）</p>
</blockquote>
<p>下面的代码是一个简单的例子（仅仅针对这个需求）：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div><div class="line">48</div><div class="line">49</div><div class="line">50</div><div class="line">51</div><div class="line">52</div><div class="line">53</div><div class="line">54</div><div class="line">55</div><div class="line">56</div><div class="line">57</div><div class="line">58</div><div class="line">59</div><div class="line">60</div><div class="line">61</div><div class="line">62</div><div class="line">63</div><div class="line">64</div></pre></td><td class="code"><pre><div class="line">import ctypes</div><div class="line">from ctypes.wintypes import *</div><div class="line">import win32clipboard</div><div class="line">from win32con import *</div><div class="line">import sys</div><div class="line"></div><div class="line">class BITMAPFILEHEADER(ctypes.Structure):</div><div class="line">    _pack_ = 1  # structure field byte alignment</div><div class="line">    _fields_ = [</div><div class="line">        (&apos;bfType&apos;, WORD),  # file type (&quot;BM&quot;)</div><div class="line">        (&apos;bfSize&apos;, DWORD),  # file size in bytes</div><div class="line">        (&apos;bfReserved1&apos;, WORD),  # must be zero</div><div class="line">        (&apos;bfReserved2&apos;, WORD),  # must be zero</div><div class="line">        (&apos;bfOffBits&apos;, DWORD),  # byte offset to the pixel array</div><div class="line">    ]</div><div class="line">SIZEOF_BITMAPFILEHEADER = ctypes.sizeof(BITMAPFILEHEADER)</div><div class="line"></div><div class="line">class BITMAPINFOHEADER(ctypes.Structure):</div><div class="line">    _pack_ = 1  # structure field byte alignment</div><div class="line">    _fields_ = [</div><div class="line">        (&apos;biSize&apos;, DWORD),</div><div class="line">        (&apos;biWidth&apos;, LONG),</div><div class="line">        (&apos;biHeight&apos;, LONG),</div><div class="line">        (&apos;biPLanes&apos;, WORD),</div><div class="line">        (&apos;biBitCount&apos;, WORD),</div><div class="line">        (&apos;biCompression&apos;, DWORD),</div><div class="line">        (&apos;biSizeImage&apos;, DWORD),</div><div class="line">        (&apos;biXPelsPerMeter&apos;, LONG),</div><div class="line">        (&apos;biYPelsPerMeter&apos;, LONG),</div><div class="line">        (&apos;biClrUsed&apos;, DWORD),</div><div class="line">        (&apos;biClrImportant&apos;, DWORD)</div><div class="line">    ]</div><div class="line">SIZEOF_BITMAPINFOHEADER = ctypes.sizeof(BITMAPINFOHEADER)</div><div class="line"></div><div class="line">win32clipboard.OpenClipboard()</div><div class="line">try:</div><div class="line">    if win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_DIB):</div><div class="line">        data = win32clipboard.GetClipboardData(win32clipboard.CF_DIB)</div><div class="line">    else:</div><div class="line">        print(&apos;clipboard does not contain an image in DIB format&apos;)</div><div class="line">        sys.exit(1)</div><div class="line">finally:</div><div class="line">    win32clipboard.CloseClipboard()</div><div class="line"></div><div class="line">bmih = BITMAPINFOHEADER()</div><div class="line">ctypes.memmove(ctypes.pointer(bmih), data, SIZEOF_BITMAPINFOHEADER)</div><div class="line"></div><div class="line">if bmih.biCompression != BI_BITFIELDS:  # RGBA?</div><div class="line">    print(&apos;insupported compression type &#123;&#125;&apos;.format(bmih.biCompression))</div><div class="line">    sys.exit(1)</div><div class="line"></div><div class="line">bmfh = BITMAPFILEHEADER()</div><div class="line">ctypes.memset(ctypes.pointer(bmfh), 0, SIZEOF_BITMAPFILEHEADER)  # zero structure</div><div class="line">bmfh.bfType = ord(&apos;B&apos;) | (ord(&apos;M&apos;) &lt;&lt; 8)</div><div class="line">bmfh.bfSize = SIZEOF_BITMAPFILEHEADER + len(data)  # file size</div><div class="line">SIZEOF_COLORTABLE = 0</div><div class="line">bmfh.bfOffBits = SIZEOF_BITMAPFILEHEADER + SIZEOF_BITMAPINFOHEADER + SIZEOF_COLORTABLE</div><div class="line"></div><div class="line">bmp_filename = &apos;clipboard.bmp&apos;</div><div class="line">with open(bmp_filename, &apos;wb&apos;) as bmp_file:</div><div class="line">    bmp_file.write(bmfh)</div><div class="line">    bmp_file.write(data)</div><div class="line"></div><div class="line">print(&apos;file &quot;&#123;&#125;&quot; created from clipboard image&apos;.format(bmp_filename))</div></pre></td></tr></table></figure></p>
</blockquote>
<p>经过测试，这一段代码成功的实现了读取剪贴板的图片并保存到本地。</p>
<h2 id="分析"><a href="#分析" class="headerlink" title="分析"></a>分析</h2><p>这段代码使用ctypes库来实现指针的功能，从而在内存中操作数据。这里定义了两个结构体，<code>BITMAPFILEHEADER</code> 和<code>BITMAPINFOHEADER</code>，于是，使用sizeof获取到了他们的大小。那么使用指针，从使用<code>GetClipboardData()</code>获取到的数据的头部开始移动，分别移动这两个结构体的大小，也就获取到了这两个结构体在内存中的数据。</p>
<p>代码中使用了<code>memmove</code>和<code>memset</code>两个内存操作的方法。从ctypes的官方文档上，我们可以看到这两个方法有如下的定义：</p>
<blockquote>
<p><code>ctypes.memmove(dst, src, count)</code></p>
<p>Same as the standard C memmove library function: copies count bytes from src to dst. dst and src must be integers or ctypes instances that can be converted to pointers.</p>
<p><code>ctypes.memset(dst, c, count)</code></p>
<p>Same as the standard C memset library function: fills the memory block at address dst with count bytes of value c. dst must be an integer specifying an address, or a ctypes instance.</p>
</blockquote>
<p>所以可以看出，代码里面的：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">bmih = BITMAPINFOHEADER()</div><div class="line">ctypes.memmove(ctypes.pointer(bmih), data, SIZEOF_BITMAPINFOHEADER)</div></pre></td></tr></table></figure></p>
<p>从内存中拷贝出来了<code>BITMAPINFOHEADER</code>这么大的一块的数据，并保存到了<code>bmih</code>这个变量中。<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">bmfh = BITMAPFILEHEADER()</div><div class="line">ctypes.memset(ctypes.pointer(bmfh), 0,     SIZEOF_BITMAPFILEHEADER)</div></pre></td></tr></table></figure></p>
<p>这一段在内存中开辟出了<code>BITMAPFILEHEADER</code>这么大一块区域，并全部填充为0.<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">bmfh.bfType = ord(&apos;B&apos;) | (ord(&apos;M&apos;) &lt;&lt; 8)</div></pre></td></tr></table></figure></p>
<p>这一行代码使用了位操作。首先<code>ord(&#39;B&#39;)</code>的值为66，换成二进制就是<code>1000010</code>；<code>ord(&#39;M&#39;)</code>的值为77，换成二进制就是<code>1001101</code>，然后向左移动8位，得到<code>100110100000000</code>，这个值再与<code>1000010</code>取位或，得到<code>100110101000010</code>。</p>
<p>最后，使用：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">bmfh.bfOffBits = SIZEOF_BITMAPFILEHEADER + SIZEOF_BITMAPINFOHEADER + SIZEOF_COLORTABLE</div></pre></td></tr></table></figure></p>
<p>拼装出头部的大小。然后以二进制方式，首先写文件头, 再写剪贴板获取到的字符串到本地的<code>.bmp</code>文件中，完成图片的生成。</p>
<h2 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h2><p>Python一些轮子确实非常好的提高了开发效率，例如PIL，三行代码实现了我的需求。Python在快速开发方面确实非常的方便，但是涉及到底层的一些操作的时候，还是不得不使用C语言的一些接口来进行内存的操作。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/12/02/opinionchange/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/12/02/opinionchange/" itemprop="url">
                  对Java与设计模式态度的转变
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-12-02T00:03:08+08:00">
                2015-12-02
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/态度/" itemprop="url" rel="index">
                    <span itemprop="name">态度</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/12/02/opinionchange/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/12/02/opinionchange/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>我一直自诩自己是Java一生黑。也从来不屑于设计模式这种东西，因为我一直认为，设计模式是为了解决Java的一些缺陷才存在的。对于Python来说，设计模式并不重要。不过最近在做一个新功能开发的时候对Java的态度有了一些改变。<more></more></p>
<p>事情的起因是在一个大概有十万行代码的框架中重构他的写Log方式。这个框架原来的Log模块是原作者自己写的，通过打开关闭文件来写Log。现在需要更换成Python的logging模块。对于一个设计相当糟糕的系统来说，要重构是很麻烦的事情。</p>
<p>在原来的Log方式中，文件的句柄通过参数的形式到处传，常常十多个文件句柄同时打开，还会互相影响，相当的混乱。最后由于需要一个地方来将所有模块的Log统一起来，还给系统的入口再加了一个入口，新的入口使用subprocess来执行</p>
<pre><code>`python xxxx`
</code></pre><p>调用原来的系统模块，并将控制台的输出重定向到文件。</p>
<p>重构的过程相当的痛苦，花了我几乎整个Sprint的时间，细节我会在后续的文章中讲到。今天我要讲的是单例模式让我对设计模式看法的一些改变。</p>
<p>对Python来说，是可以将一个类的实例通过参数的形式直接传递的，所以看起来似乎Python并不需要单例模式。对于新写的代码来说，这当然没有什么问题。但是对已有的代码添加新的功能的时候，这个方法就不太适用了。</p>
<p>如果原来一个方法，它的参数已经有七八个了，这个时候你在加一个参数上去，并不确定会产生什么样的影响。</p>
<p>例如：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">def foo(a, b, c):</div><div class="line">		do()</div><div class="line">		some()</div><div class="line">		thing()</div></pre></td></tr></table></figure>
<p>如果想加一个新的参数，但是又不影响其他的已经在调用这个方法的地方，当然可以这样写：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line"> def foo(a, b, c, d=None):</div><div class="line">do()</div><div class="line">some()</div><div class="line">thing()</div></pre></td></tr></table></figure>
<p>如果代码量不大，当然没有问题，但是如果代码量大起来了，这样就会出问题。</p>
<p>例如，现在有一个实例instanceA 在module1 中初始化了，而module3如果想使用instanceA，就必须想办法让instanceA 传递到module3去。但是原本module3和module1并没有什么关系，现在因为需要这个instanceA, 怎么办呢？这个时候发现instanceA 被传递到了module2, 然后又从module2传递到了module4, 最后在module4里面终于找到了一个地方，可以强行把instanceA和module3联系在一起。这样一来，这几个module他们被强制耦合在了一起。</p>
<p>对于程序来说，耦合程度应该越低越好，所以这样的做法，非常不妥。</p>
<p>于是，这个时候单例模式就出现了。</p>
<p>对于module3来说，如果他想要instanceA, 他完全可以再重新new一个，由于是单例模式，于是新的instance 和instanceA是完全一样的。这样就并没有增加多余的耦合关系。</p>
<p>另外，在极客学院录课的时候，最近有涉及到程序集之间通信。这个时候偶然发现设计模式里面的观察者模式可以对我的设计起到相当大的帮助。</p>
<p>就像以前某人说的，你要讨厌一个东西，你必须要比喜欢这个东西的人更了解它。而我以前讨厌Java仅仅是因为Java看起来太罗嗦，太多冗余代码。而我不屑于设计模式，更是因为它是从Java来的。</p>
<p>现在看起来，态度真的应该改变一下了。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/10/07/lxmlencoding/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/10/07/lxmlencoding/" itemprop="url">
                  使用etree.HTML的编码问题
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-10-07T17:56:47+08:00">
                2015-10-07
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/10/07/lxmlencoding/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/10/07/lxmlencoding/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <h2 id="出现问题"><a href="#出现问题" class="headerlink" title="出现问题"></a>出现问题</h2><p>今天指导一个学生爬取新浪体育手机版的时候，发现lxml.etree.HTML处理网页源代码会默认修改编码，导致打印出来的内容为乱码。爬取的网址为：<a href="http://sports.sina.cn/nba/rockets/2015-10-07/detail-ifximrxn8235561.d.html?vt=4&amp;pos=10" title="http://sports.sina.cn/nba/rockets/2015-10-07/detail-ifximrxn8235561.d.html?vt=4&amp;pos=10" target="_blank" rel="external">http://sports.sina.cn/nba/rockets/2015-10-07/detail-ifximrxn8235561.d.html?vt=4&amp;pos=10</a></p>
<p>首先导入我们需要用到的库文件，然后设置环境：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">#-*_coding:utf8-*-</div><div class="line">import requests</div><div class="line">from lxml import etree</div><div class="line">import sys</div><div class="line">reload(sys)</div><div class="line">sys.setdefaultencoding(&quot;utf-8&quot;)</div></pre></td></tr></table></figure>
<p>然后获取网页的源代码：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line"></div><div class="line">r = requests.get(url=&apos;http://sports.sina.cn/nba/rockets/2015-10-07/detail-ifximrxn8235561.d.html?vt=4&amp;pos=10&apos;)# 最基本的GET请求</div><div class="line">r.encoding = &apos;utf-8&apos;</div><div class="line">r = r.content</div><div class="line">print r</div></pre></td></tr></table></figure></p>
<p>打印出网页源代码，发现中文是乱码，如图：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding1.gif" alt=""></p>
<p>这是小问题，使用<a href="http://blog.kingname.info/2014/12/14/Python%E5%AD%97%E7%AC%A6%E7%BC%96%E7%A0%81%E7%9A%84%E4%B8%80%E4%B8%AA%E7%9B%B8%E5%AF%B9%E4%B8%87%E8%83%BD%E7%9A%84%E5%A4%84%E7%90%86%E6%96%B9%E6%B3%95/" target="_blank" rel="external">Python字符编码的一个相对万能的处理方法</a>这篇文章中讲解的方法，轻松解决。</p>
<p>将：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">r = r.content</div></pre></td></tr></table></figure></p>
<p>修改为:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">r = r.content.decode(&apos;utf-8&apos;).encode(&apos;gbk&apos;)</div></pre></td></tr></table></figure></p>
<p>可以正常显示中文，如图：<br><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding2.png" alt=""></p>
<p>接下来，使用etree.HTML处理源代码，然后使用Xpath提取内容，一切似乎看起来轻车熟路。</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding3.png" alt=""></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">contentTree = etree.HTML(r)</div><div class="line">title = contentTree.xpath(<span class="string">'//h1[@class="art_title_h1"]/text()'</span>)</div><div class="line"><span class="keyword">print</span> title[<span class="number">0</span>]</div></pre></td></tr></table></figure>
<p>但是当我打印出来，才发现问题没有这么简单。如图：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding4.png" alt=""></p>
<p>这个时候，我发现使用<a href="http://blog.kingname.info/2014/12/14/Python%E5%AD%97%E7%AC%A6%E7%BC%96%E7%A0%81%E7%9A%84%E4%B8%80%E4%B8%AA%E7%9B%B8%E5%AF%B9%E4%B8%87%E8%83%BD%E7%9A%84%E5%A4%84%E7%90%86%E6%96%B9%E6%B3%95/" target="_blank" rel="external">Python字符编码的一个相对万能的处理方法</a>讲到的办法已经不能解决问题了。</p>
<p>通过调试，我发现抓取到的内容是乱码：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding5.png" alt=""></p>
<h2 id="解决办法"><a href="#解决办法" class="headerlink" title="解决办法"></a>解决办法</h2><h3 id="使用Scrapy"><a href="#使用Scrapy" class="headerlink" title="使用Scrapy"></a>使用Scrapy</h3><p>使用Scrapy的Xpath，正常提取需要的内容：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding6.png" alt=""></p>
<h3 id="继续用etree"><a href="#继续用etree" class="headerlink" title="继续用etree"></a>继续用etree</h3><p>实际上，Scrapy的Xpath底层还是调用的lxml,那为什么它可以，而我直接使用lxml的etree.HTML处理源代码然后Xpath提取内容就出乱码呢？</p>
<p>显然这应该是编码的问题，在使用:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">etree.HTML(r)</div></pre></td></tr></table></figure>
<p>处理源文件的时候，由于没有指定编码，所以它使用了一个默认编码，从而导致和UTF-8冲突，产生乱码。</p>
<p>经过查阅lxml.etree.HTML的文档，我发现etree.HTML有一个参数是parser,这个参数不是必须的，因此省略以后它就会自动使用一个默认的parser。既然如此，那我手动指定一个：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">contentTree = etree.HTML(r, parser=etree.HTMLParser(encoding=<span class="string">'utf-8'</span>))</div></pre></td></tr></table></figure>
<p>这里我指定了etree.HTMLParser来作为一个parser,同时，etree.HTMLParser可以接受编码作为参数。于是我指定为UTF-8。</p>
<p>运行看看效果：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding7.png" alt=""></p>
<p>继续报错，但是出错信息改变了，提示utf8不能解码。请注意第11行，现在源代码是gbk编码，所以使用UTF-8不能解码。于是可以把第11行重新改回原来的样子：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">r = r.content</div></pre></td></tr></table></figure>
<p>再一次运行，发现正常抓取信息：</p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/lxmlencoding8.png" alt=""></p>
<h2 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h2><p>这一次的问题提示我们：遇到问题，通过经验解决不了的时候，请回归文档。</p>
<p><em>原文发表在：<a href="http://blog.kingname.info/2015/10/07/lxmlencoding/" target="_blank" rel="external">http://blog.kingname.info/2015/10/07/lxmlencoding/</a>转载请注明出处！</em></p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/07/19/mycodestyle/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/07/19/mycodestyle/" itemprop="url">
                  我的编码规范
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-07-19T16:31:54+08:00">
                2015-07-19
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/07/19/mycodestyle/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/07/19/mycodestyle/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <h2 id="开篇闲扯"><a href="#开篇闲扯" class="headerlink" title="开篇闲扯"></a>开篇闲扯</h2><p>距离上一次更新已经有快三个月了，这三个月我都在干什么呢？首先是继续实习，因为一些原因，本来计划的三个月实习变成了四个月。在实习的过程中逐步接手了一些维护的工作，并提交了不少代码。7月6号正式入职以后，我加入了一个Amazing的新项目的创建，目前我们做出来的Demo真是非常的Amazing。不过可惜的是，这样一个项目在最近几年是不会对外公布的。</p>
<p>在6月之前，每晚回家就是做毕设了，这段时间休息的都比较晚，头发都白了很多。</p>
<p>还有就是给极客学院录课，第一个系列课程《定向爬虫入门》已经全部录制完毕了，一共8课，大家可以在about me 里面找到前7课的链接，最后一课目前正在后期制作当中，上线以后会更新链接。接下来我会讲解远程控制软件的编写，欢迎大家在极客学院关注我，我的名字叫kingname~</p>
<p>闲话完毕，那我们开始正题。</p>
<p>##本题由来<br>其实以前我是不太注意编码风格的，觉得程序可以运行了就行。甚至网上还有一种说法：</p>
<blockquote>
<p>如果你急急忙忙写一个程序，都没有注重编码规范，程序可能一次就跑通了；但是如果你仔细编码，而且还把单元测试做的非常完善，很有可能程序会出现大bug,而且还难以找到原因。</p>
</blockquote>
<p>然而在我实习的过程中，读了公司的很多代码，才发现编码规范是如此的重要，以至于可以显著影响对代码的理解。而且由于在部门里面code review做的比较严格，因此代码的风格不统一的话，是不能merge的，于是在多次提交代码以后渐渐的总结出了一些规律。</p>
<p>##变量命名<br>在我大一学编程的时候，老师告诉我们变量命名最后使用匈牙利命名法，例如，经常出现hszStr、istuNum之类的变量名，如果对代码不熟悉的话，都不知道是什么意思。</p>
<p>现在我们并不使用匈牙利命名法，而是直接将变量的含义通过单纯完整拼写处理。例如：</p>
<blockquote>
<p>deviceSerialDict： 用于保存设备串号的字典<br>deviceSerialDictList： 用于保存设备串号字典的列表<br>unusedSerialNumList： 未使用的序列号列表</p>
</blockquote>
<p>我们约定：</p>
<ul>
<li>变量名首字母小写</li>
<li>多个单词的构成的变量，从第二个开始首字母大写</li>
<li>缩写词做单词处理，例如ACM在变量名中应该写为Acm而不是ACM</li>
<li>常量大写，每个单词使用_下划线分割</li>
<li>不允许出现Magic Number</li>
<li>使用名词或者动宾结构</li>
</ul>
<p>##类与方法</p>
<p>使用面向对象编程，这一点在我做毕业设计的程序时，帮了我大忙，大大减轻了我的工作压力。</p>
<p>我们的风格其实有点像Java了，文件名和类名相同，如果一个文件里面有多个类，那文件名与主要的类名相同。</p>
<p>我们约定：</p>
<ul>
<li>非私有的方法，首字母小写，动宾结构，例如getSerialNum</li>
<li>对于和类本身没有什么大关系的方法，要使用@staticmethod</li>
<li>只有类自己使用的方法需要设为私有，也就是名称前加两个下划线</li>
</ul>
<p>##代码细节</p>
<ul>
<li>逗号后面要有一个空格</li>
<li>每一行代码后面不能有空格</li>
<li>文件的最后要有一个空行</li>
<li>等号左右都需要有空格，但是作为参数的时候不需要空格</li>
<li>方法的参数大于6个，则使用列表或者字典来传递</li>
</ul>
<p>##总结</p>
<p>挂一漏万，以上的规范使用Pylint都可以帮忙检查出来。没有说到的地方请大家补充，除了以上的规范以外，还有Python自己本身的编码规范，请戳-&gt;<a href="https://www.python.org/dev/peps/pep-0008/" target="_blank" rel="external">https://www.python.org/dev/peps/pep-0008/</a>。 </p>
<p>最近重构了MCC,请大家对比</p>
<p><a href="https://github.com/kingname/MCC" target="_blank" rel="external">https://github.com/kingname/MCC</a> </p>
<p>与 重构之前 </p>
<p><a href="https://github.com/kingname/MCC/tree/c806c3ccfd0c0585d51caa9f85e6867e0f3ee8cb" target="_blank" rel="external">https://github.com/kingname/MCC/tree/c806c3ccfd0c0585d51caa9f85e6867e0f3ee8cb</a></p>
<p>的区别。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/04/29/lxmloldbug/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/04/29/lxmloldbug/" itemprop="url">
                  requests的content与text导致lxml的解析问题
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-04-29T22:49:31+08:00">
                2015-04-29
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/经验/" itemprop="url" rel="index">
                    <span itemprop="name">经验</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/04/29/lxmloldbug/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/04/29/lxmloldbug/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>最近实习工作任务比较重，又在做数据挖掘的教学工作，同时还在做毕设，所以博客更新比较慢。不过最近肯定会有大动作。</p>
<p>闲话毕，转正题。在制作新浪微博模拟登录的部分时，遇到了一些问题。</p>
<p>我使用requests获取了新浪微博的源代码，通过lxml库的etree.HTML来处理一段网页源代码，从而生成一个可以被xpath解析的对象。<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">selector = etree.HTML(html)</div></pre></td></tr></table></figure></p>
<p>遇到报错：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">selector = etree.HTML(html)</div><div class="line">  File &quot;lxml.etree.pyx&quot;, line 2953, in lxml.etree.HTML (src\lxml\lxml.etree.c:66734)</div><div class="line">  File &quot;parser.pxi&quot;, line 1780, in lxml.etree._parseMemoryDocument (src\lxml\lxml.etree.c:101591)</div><div class="line">ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.</div></pre></td></tr></table></figure></p>
<p>根据报错信息推测，可能是因为不支持编码声明的Unicode字符串。Google发现这个问题在2012年就已经有人提交给作者了，但是一直没有被修复。地址在-&gt;<a href="https://gist.github.com/karlcow/3258330" target="_blank" rel="external">https://gist.github.com/karlcow/3258330</a></p>
<p>不过下面的人也给出了解决办法：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">html = bytes(bytearray(html, encoding=&apos;utf-8&apos;))</div><div class="line">selector = etree.HTML(html)</div></pre></td></tr></table></figure></p>
<p>首先将源代码转化成比特数组，然后再将比特数组转化成一个比特对象。这样就可以绕过这个bug。</p>
<p>然而，又有人认为这不是一个bug, 所以一直没有被修复。这是由于，我获取源代码是使用r.text<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">html = requests.get(&apos;xxxxxx&apos;,cookies=cookies).text</div></pre></td></tr></table></figure></p>
<p>而如果使用r.content：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">html = requests.get(&apos;xxxxxx&apos;,cookies=cookies).content</div></pre></td></tr></table></figure></p>
<p>就不会报错。</p>
<p>那r.text与r.content有什么区别呢？分析requests的源代码发现，r.text返回的是Unicode型的数据，而使用r.content返回的是bytes型的数据。也就是说，在使用r.content的时候，他已经只带了<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">html = bytes(bytearray(html, encoding=&apos;utf-8&apos;))</div></pre></td></tr></table></figure></p>
<p>这样一个转化了。</p>
<p>最近CentOS都声明放弃Python2了，编码问题确实浪费了很多时间，等空下来转Python3吧~</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/03/04/pythonunittest2/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/03/04/pythonunittest2/" itemprop="url">
                  Python的单元测试（二）
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-03-04T19:08:20+08:00">
                2015-03-04
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/03/04/pythonunittest2/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/03/04/pythonunittest2/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>在<a href="http://kingname.info/2015/02/27/Python%E7%9A%84%E5%8D%95%E5%85%83%E6%B5%8B%E8%AF%95/">Python的单元测试（一）</a>中，我们讲了单元测试的概念以及一个简单的单元测试例子。</p>
<p>在这个例子中，只有三个函数，于是可以把每个函数的输出结果打印到屏幕上，再用肉眼去看结果是否符合预期。然而假设有一个程序，有二十个类，每个类又有几十个函数，有些函数的输出结果还多达几十行，在这种情况下，肉眼如何看得出？</p>
<p>当然你可以使用if判断<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">if</span> 输出结果 == 预期结果:</div><div class="line">	<span class="keyword">return</span> <span class="keyword">True</span></div><div class="line"><span class="keyword">else</span>:</div><div class="line">	<span class="keyword">print</span> <span class="string">u'不相等'</span></div></pre></td></tr></table></figure></p>
<p>这个时候，你发现，程序有几个函数，后三行就要重复几次，本着代码简洁的原则，你把这个判断的过程写到一个函数中：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line"><span class="function"><span class="keyword">def</span> <span class="title">isequal</span><span class="params">(output,right_output)</span>:</span></div><div class="line">	<span class="keyword">if</span> output == right_output:</div><div class="line">		<span class="keyword">return</span> <span class="keyword">True</span></div><div class="line">	<span class="keyword">else</span>:</div><div class="line">		<span class="keyword">print</span> <span class="string">u'不相等'</span></div></pre></td></tr></table></figure></p>
<p>那么恭喜你，你步入正规了，然而，这一切已经有人为你做好了。欢迎unittest模块出场。</p>
<blockquote>
<p>unittest supports test automation, sharing of setup and shutdown code for tests, aggregation of tests into collections, and independence of the tests from the reporting framework. The unittest module provides classes that make it easy to support these qualities for a set of tests.</p>
</blockquote>
<p>Python的官方文档这样写到，unittest支持自动化测试，测试的安装分享和关闭代码……</p>
<p>一句话说来，就是，unittest很好用。</p>
<p>还是用上一次的readandadd.py来演示unittest的基本用法,首先介绍unittest的一个函数，assertEqual(first,second),这个函数的作用是检查变量first的值与second的值是否相等，如果不相等就抛出错误。 </p>
<p>先创建utest.py文件，输入以下代码并运行：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div></pre></td><td class="code"><pre><div class="line">#-*-coding:utf-8-*-</div><div class="line">import unittest</div><div class="line">import readandadd</div><div class="line"></div><div class="line">class basictest(unittest.TestCase): #类名可以随便取</div><div class="line">    def testread(self): #每个函数都要以test开头</div><div class="line">        output = readandadd.read(&apos;1.txt&apos;)</div><div class="line">        self.assertEqual(output,&apos;2,3&apos;)</div><div class="line"></div><div class="line">    def testgetnum(self):</div><div class="line">        output = readandadd.getnum(&apos;2,3&apos;)</div><div class="line">        self.assertEqual(output,[&apos;2&apos;, &apos;3&apos;])</div><div class="line"></div><div class="line">    def testaddnum(self):</div><div class="line">        output = readandadd.addnum([2,3])</div><div class="line">        self.assertEqual(output,5)</div><div class="line"></div><div class="line">if __name__ == &apos;__main__&apos;:</div><div class="line">    unittest.main()</div></pre></td></tr></table></figure></p>
<p>运行结果如下：<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">...</div><div class="line">----------------------------------------------------------------------</div><div class="line">Ran 3 tests <span class="keyword">in</span> 0.001s</div><div class="line"></div><div class="line">OK</div></pre></td></tr></table></figure></p>
<p>你也许会说，就一个ok，什么都没有啊。那我先把testread()函数下面的<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">self.assertEqual(output,<span class="string">'2,3'</span>)</div></pre></td></tr></table></figure></p>
<p>改为<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">self.assertEqual(output,<span class="string">'2,4'</span>)</div></pre></td></tr></table></figure></p>
<p>在运行utest.py看看输出结果如何：<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div></pre></td><td class="code"><pre><div class="line">..F</div><div class="line">======================================================================</div><div class="line">FAIL: testread (__main__.basictest)</div><div class="line">----------------------------------------------------------------------</div><div class="line">Traceback (most recent call last):</div><div class="line">  File <span class="string">"E:/mystuff/unitest/utest.py"</span>, line 8, <span class="keyword">in</span> testread</div><div class="line">    self.assertEqual(output,<span class="string">'2,4'</span>)</div><div class="line">AssertionError: <span class="string">'2,3'</span> != <span class="string">'2,4'</span></div><div class="line"></div><div class="line">----------------------------------------------------------------------</div><div class="line">Ran 3 tests <span class="keyword">in</span> 0.000s</div><div class="line"></div><div class="line">FAILED (failures=1)</div></pre></td></tr></table></figure></p>
<p>这里准确的找出了错误的位置和错误的具体内容。注意看最上面，有个<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">..F</div></pre></td></tr></table></figure></p>
<p>猜测它可能是标示错误的位置。保持testread的错误不改，再把testgetnum()函数中的以下内容<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">self.assertEqual(output,[<span class="string">'2'</span>, <span class="string">'3'</span>])</div></pre></td></tr></table></figure></p>
<p>改为<br><figure class="highlight python"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">self.assertEqual(output,[<span class="string">'2'</span>, <span class="string">'6'</span>])</div></pre></td></tr></table></figure></p>
<p>再运行utest.py程序，输出结果如下：<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div></pre></td><td class="code"><pre><div class="line">.FF</div><div class="line">======================================================================</div><div class="line">FAIL: testgetnum (__main__.basictest)</div><div class="line">----------------------------------------------------------------------</div><div class="line">Traceback (most recent call last):</div><div class="line">  File <span class="string">"E:/mystuff/unitest/utest.py"</span>, line 12, <span class="keyword">in</span> testgetnum</div><div class="line">    self.assertEqual(output,[<span class="string">'2'</span>, <span class="string">'6'</span>])</div><div class="line">AssertionError: Lists differ: [<span class="string">'2'</span>, <span class="string">'3'</span>] != [<span class="string">'2'</span>, <span class="string">'6'</span>]</div><div class="line"></div><div class="line">First differing element 1:</div><div class="line">3</div><div class="line">6</div><div class="line"></div><div class="line">- [<span class="string">'2'</span>, <span class="string">'3'</span>]</div><div class="line">?        ^</div><div class="line"></div><div class="line">+ [<span class="string">'2'</span>, <span class="string">'6'</span>]</div><div class="line">?        ^</div><div class="line"></div><div class="line"></div><div class="line">======================================================================</div><div class="line">FAIL: testread (__main__.basictest)</div><div class="line">----------------------------------------------------------------------</div><div class="line">Traceback (most recent call last):</div><div class="line">  File <span class="string">"E:/mystuff/unitest/utest.py"</span>, line 8, <span class="keyword">in</span> testread</div><div class="line">    self.assertEqual(output,<span class="string">'2,4'</span>)</div><div class="line">AssertionError: <span class="string">'2,3'</span> != <span class="string">'2,4'</span></div><div class="line"></div><div class="line">----------------------------------------------------------------------</div><div class="line">Ran 3 tests <span class="keyword">in</span> 0.001s</div><div class="line"></div><div class="line">FAILED (failures=2)</div></pre></td></tr></table></figure></p>
<p>可以看出，这里分别把两个错误显示了出来。并且第一行变成了<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">.FF</div></pre></td></tr></table></figure></p>
<p>所以，第一行的内容应该从右往左读，它标明错误函数在所有函数的相对位置。</p>
<p>现在再把testread()和testgetnum()改回去，再看看全部正确的输出：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">...</div><div class="line">----------------------------------------------------------------------</div><div class="line">Ran 3 tests in 0.000s</div><div class="line"></div><div class="line">OK</div></pre></td></tr></table></figure></p>
<p>印证了那句话，没有消息就是最好的消息。</p>
<p>这篇文章介绍了单元测试模块unittest的assertEqual的基本用法，下一篇文章将要更加全面的介绍unittest模块。</p>
<hr>
<p>本文首发地址:<a href="http://kingname.info/2015/03/04/pythonunittest2/">http://kingname.info/2015/03/04/pythonunittest2/</a></p>
<p>作者：青南　</p>
<p>转载请注明出处。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/02/27/Python的单元测试/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/02/27/Python的单元测试/" itemprop="url">
                  Python的单元测试（一）
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-02-27T22:50:47+08:00">
                2015-02-27
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/02/27/Python的单元测试/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/02/27/Python的单元测试/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>测试驱动的软件开发方式可以强迫程序员在开发程序的时候使程序的函数之间实现高内聚，低耦合。这样的方式可以降低函数之间的依赖性，方便后续的修改，增加功能和维护。</p>
<p>说<strong>一个函数</strong>高内聚，就是指这个函数专注于实现单一的任务，不会做除了生产这个任务以外的其他事情。可以想象一个人，他把自己关在一个小房子里面生产东西，只留两扇窗户，他需要什么材料，你就从小窗户给他送进去（参数），他做好了东西，就给你从另一个窗户里面送出来（return），他不会说，我要生产一个轮子，但是我首先需要一个女人进来，他不会说，这是计划的一部分。</p>
<p>说<strong>几个函数</strong>是低耦合的，就是指他们的依赖性小。他们就像是葫芦娃，每个都有自己独特的能力，可以自己单干，在关键的时候还可以合体，变成小金刚。他们就像积木一样，各有各的功能，需要使用的时候直接组合在一起就可以了。</p>
<p>使用测试驱动开发，每一个测试只测试一个功能，这样就可以迫使函数把自己独立出来，尽量减少和其他函数的依赖。</p>
<p>例如，有一个文件1.txt，他的内容是两个数字，使用逗号隔开。形如“2,4”（不包括外侧双引号，下同）。我要写一个程序readandadd.py，读取硬盘上的1.txt文件，然后把这个文件的内容打印到屏幕上。</p>
<p>不规范的写法一：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">f= open(&apos;1.txt&apos;,&apos;r&apos;)</div><div class="line">b = f.read().split(&apos;,&apos;)</div><div class="line">f.close()</div><div class="line">print int(b[0])+int(b[1])</div></pre></td></tr></table></figure></p>
<p>不规范写法二：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">def A():</div><div class="line">	f= open(&apos;1.txt&apos;,&apos;r&apos;)</div><div class="line">	b = f.read().split(&apos;,&apos;)</div><div class="line">	f.close()</div><div class="line">	print int(b[0])+int(b[1])</div><div class="line">A()</div></pre></td></tr></table></figure></p>
<p>比较规范的写法：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div></pre></td><td class="code"><pre><div class="line">def read(filename):</div><div class="line">    f= open(filename,&apos;r&apos;)</div><div class="line">    info = f.read()</div><div class="line">    f.close()</div><div class="line">    return info</div><div class="line"></div><div class="line">def getnum(info):</div><div class="line">    twonum = info.split(&apos;,&apos;)</div><div class="line">    return twonum</div><div class="line"></div><div class="line">def addnum(twonum):</div><div class="line">    return int(twonum[0])+int(twonum[1])</div><div class="line"></div><div class="line">if __name__ == &apos;__main__&apos;:</div><div class="line">        info = read(&apos;1.txt&apos;)</div><div class="line">        twonum = getnum(info)</div><div class="line">        result = addnum(twonum)</div><div class="line">		print result</div></pre></td></tr></table></figure></p>
<p>这样写的好处是，如果想测试读文件的功能，就只需要测试read()函数，如果想测试把两个数分开的功能，就只需要测试getnum()函数。而相反，在不规范写法二中，虽然只想测试两个数字相加的功能，可是却不得不首先打开文件并读取文件然后把数字分开。</p>
<p>继续回到比较规范的写法当中，我相信很多人写完read()函数以后，肯定会输入如下代码：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">def read(filename):</div><div class="line">    f= open(filename,&apos;r&apos;)</div><div class="line">    info = f.read()</div><div class="line">    f.close()</div><div class="line">    return info</div><div class="line"></div><div class="line">print read(&apos;1.txt&apos;)</div></pre></td></tr></table></figure></p>
<p>然后运行程序，发现正常打印出’2,3’以后，再开始写getnum()函数。写完getnum以后，测试getnum()函数没问题以后再开始写然后测试addnum()函数。最后测试整个程序的功能。</p>
<p>其实这个过程，已经就是在做单元测试了。然而这样操作的弊端是什么？如果整体程序已经写好了，之前做测试点代码也就删除了。那么如果突然把程序做了修改。例如1.txt里面数字的分隔从1个逗号变成了空格，或者变成了3个数字，那必然要修改getnum()，但是又如何测试修改的部分呢？还要把不相干的代码给注释掉。不仅麻烦，而且容易出错。</p>
<p>现在，把测试的代码单独独立出来。会有什么效果呢？尝试创建一个test.py程序，代码如下：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">import readandadd</div><div class="line"></div><div class="line">def testread():</div><div class="line">    print &apos;read:&apos;,readandadd.read(&apos;1.txt&apos;)</div><div class="line"></div><div class="line">def testgetnum():</div><div class="line">    print &apos;getnum:&apos;,readandadd.getnum(&apos;2,3&apos;)</div><div class="line"></div><div class="line">def testaddnum():</div><div class="line">    print &apos;addnum:&apos;,readandadd.addnum([2,3])</div><div class="line"></div><div class="line">if __name__ == &apos;__main__&apos;:</div><div class="line">    testread()</div><div class="line">    testgetnum()</div><div class="line">    testaddnum()</div></pre></td></tr></table></figure></p>
<p>运行test.py以后输出结果如下：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">read: 2,3</div><div class="line">getnum: [&apos;2&apos;, &apos;3&apos;]</div><div class="line">addnum: 5</div></pre></td></tr></table></figure></p>
<p>每一个函数的输出结果一目了然，而且在修改了readandadd.py的函数以后，重新运行test.py就可以知道输出结果有没有符合预期。</p>
<p>当然，这里这个例子非常的简单，因此可以人工通过观察test.py的输出结果来确定是否符合预期，那对于大量的函数的测试，难道也要让肉眼来看吗？当然不是。于是，下一篇文章将会介绍Python的单元测试unittest。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/01/30/未知道——广场式的真匿名交流网站/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/01/30/未知道——广场式的真匿名交流网站/" itemprop="url">
                  未知道——广场式的真匿名交流网站（一）
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-01-30T15:24:35+08:00">
                2015-01-30
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/未知道/" itemprop="url" rel="index">
                    <span itemprop="name">未知道</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/01/30/未知道——广场式的真匿名交流网站/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/01/30/未知道——广场式的真匿名交流网站/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>最近在学习Flask，本着学以致用的原则，开发了“未知道”这个网站。未知道的宗旨是，不注册，不审核，不删帖，无私聊，不记录任何信息。是一个派对式的或者说广场式的真·匿名聊天环境。</p>
<p>网站地址请戳-&gt;<a href="http://unknown.kingname.info/" target="_blank" rel="external">http://unknown.kingname.info/</a></p>
<p><img src="http://7sbpmp.com1.z0.glb.clouddn.com/unknown/QQ截图20150130152847.jpg" alt=""></p>
<p>网站上线首日访问量突破2000，上线第二日访问量突破10000。</p>
<p>现在到处都在要求实名制，请珍惜这少有的一片匿名的净土。</p>
<p>这个网站目前仍然在每天增加新的功能，我会开一个系列文章，用以记录在这个网站的开发过程中遇到的问题以及解决方案。请大家保持关注。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/01/17/xpath提取多个标签下的text/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/01/17/xpath提取多个标签下的text/" itemprop="url">
                  xpath提取多个标签下的text
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-01-17T16:01:07+08:00">
                2015-01-17
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/01/17/xpath提取多个标签下的text/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/01/17/xpath提取多个标签下的text/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>在写爬虫的时候，经常会使用xpath进行数据的提取，对于如下的代码：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">&lt;div id=&quot;test1&quot;&gt;大家好！&lt;/div&gt;</div></pre></td></tr></table></figure></p>
<p>使用xpath提取是非常方便的。假设网页的源代码在selector中：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">data = selector.xpath(&apos;//div[@id=&quot;test1&quot;]/text()&apos;).extract()[0]</div></pre></td></tr></table></figure></p>
<p>就可以把“大家好！”提取到data变量中去。</p>
<p>然而如果遇到下面这段代码呢？<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">&lt;div id=&quot;test2&quot;&gt;美女，&lt;font color=red&gt;你的微信是多少？&lt;/font&gt;&lt;div&gt;</div></pre></td></tr></table></figure></p>
<p>如果使用：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">data = selector.xpath(&apos;//div[@id=&quot;test2&quot;]/text()&apos;).extract()[0]</div></pre></td></tr></table></figure></p>
<p>只能提取到“美女，”；</p>
<p>如果使用：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">data = selector.xpath(&apos;//div[@id=&quot;test2&quot;]/font/text()&apos;).extract()[0]</div></pre></td></tr></table></figure></p>
<p>又只能提取到“你的微信是多少？”</p>
<p>可是我本意是想把“美女，你的微信是多少？”这一整个句子提取出来。</p>
<p>这还不是最糟糕的，还有第三段代码：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">&lt;div id=&quot;test3&quot;&gt;我左青龙，&lt;span id=&quot;tiger&quot;&gt;右白虎，&lt;ul&gt;上朱雀，&lt;li&gt;下玄武。&lt;/li&gt;&lt;/ul&gt;老牛在当中，&lt;/span&gt;龙头在胸口。&lt;div&gt;</div></pre></td></tr></table></figure></p>
<p>而且内部的标签还不固定，如果我有一百段这样类似的html代码，又如何使用xpath表达式，以最快最方便的方式提取出来？</p>
<p>我差一点就去用正则表达式替换了。还好我去Stack Overflow上面提了问。于是很快就有人给我解答了。</p>
<p><strong>使用xpath的string(.)</strong></p>
<p>以第三段代码为例：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">data = selector.xpath(&apos;//div[@id=&quot;test3&quot;]&apos;)</div><div class="line">info = data.xpath(&apos;string(.)&apos;).extract()[0]</div></pre></td></tr></table></figure></p>
<p>这样，就可以把“我左青龙，右白虎，上朱雀，下玄武。老牛在当中，龙头在胸口”整个句子提取出来，赋值给info变量。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="https://kingname.info/2015/01/08/清空Github上某个文件的历史/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Kingname">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="Kingname">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="Kingname" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2015/01/08/清空Github上某个文件的历史/" itemprop="url">
                  清空Github上某个文件的历史
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2015-01-08T16:04:53+08:00">
                2015-01-08
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/经验/" itemprop="url" rel="index">
                    <span itemprop="name">经验</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <a href="/2015/01/08/清空Github上某个文件的历史/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2015/01/08/清空Github上某个文件的历史/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          

          
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>文章首发地址：<a href="http://kingname.info">http://kingname.info</a></p>
<p>今天在Github更新代码的时候，不小心把Gmail私钥文件更新上去了。即便我立刻删除了这个文件，可是在版本历史里面仍然可以看到这个文件的内容。这可把我吓坏了。</p>
<p>Google一圈以后，终于找到了解决办法。把某个文件的历史版本全部清空。</p>
<p>首先cd 进入项目文件夹下，然后执行以下代码：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div></pre></td><td class="code"><pre><div class="line">git filter-branch --force --index-filter &apos;git rm --cached --ignore-unmatch 文件名&apos; --prune-empty --tag-name-filter cat -- --all</div><div class="line"></div><div class="line">git push origin master --force</div><div class="line"></div><div class="line">rm -rf .git/refs/original/</div><div class="line"></div><div class="line">git reflog expire --expire=now --all</div><div class="line"></div><div class="line">git gc --prune=now</div><div class="line"></div><div class="line">git gc --aggressive --prune=now</div></pre></td></tr></table></figure></p>
<p>虽然不知道他们的作用是什么，不过真的解决了我的问题。看起来，以前我说我熟练掌握git，真是自不量力。</p>

          
        
      
    </div>

    <div>
      
    </div>

    <div>
      
    </div>


    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
  </section>

  
  <nav class="pagination">
    <a class="extend prev" rel="prev" href="/page/2/"><i class="fa fa-angle-left"></i></a><a class="page-number" href="/">1</a><a class="page-number" href="/page/2/">2</a><span class="page-number current">3</span><a class="page-number" href="/page/4/">4</a><a class="page-number" href="/page/5/">5</a><a class="extend next" rel="next" href="/page/4/"><i class="fa fa-angle-right"></i></a>
  </nav>



          </div>
          


          
        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      

      <section class="site-overview sidebar-panel sidebar-panel-active">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/avatar.png"
               alt="Kingname" />
          <p class="site-author-name" itemprop="name">Kingname</p>
          <p class="site-description motion-element" itemprop="description">爬虫工程师，极客学院讲师，Python, Scrapy, MongoDB, Redis, Pandas, Golang。</p>
        </div>
        <nav class="site-state motion-element">
          <div class="site-state-item site-state-posts">
            <a href="/archives">
              <span class="site-state-item-count">42</span>
              <span class="site-state-item-name">日志</span>
            </a>
          </div>

          
            <div class="site-state-item site-state-categories">
              
                <span class="site-state-item-count">13</span>
                <span class="site-state-item-name">分类</span>
              
            </div>
          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">66</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        
          <div class="feed-link motion-element">
            <a href="/atom.xml" rel="alternate">
              <i class="fa fa-rss"></i>
              RSS
            </a>
          </div>
        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/kingname" target="_blank" title="Github">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Github
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="mailto:contact@kingname.info" target="_blank" title="Mail">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Mail
                </a>
              </span>
            
          
        </div>

        
        

        
        
          <div class="links-of-blogroll motion-element links-of-blogroll-inline">
            <div class="links-of-blogroll-title">
              <i class="fa  fa-fw fa-globe"></i>
              Friends
            </div>
            <ul class="links-of-blogroll-list">
              
                <li class="links-of-blogroll-item">
                  <a href="http://chuangzaoshi.com/" title="创造狮" target="_blank">创造狮</a>
                </li>
              
            </ul>
          </div>
        

        


      </section>

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy; 
  <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Kingname</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Mist
  </a>
</div>


        

        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  



  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.0"></script>



  
  

  

  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.0"></script>



  





  
  

  
  
    <script type="text/x-mathjax-config">
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
          processEscapes: true,
          skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
        }
      });
    </script>

    <script type="text/x-mathjax-config">
      MathJax.Hub.Queue(function() {
        var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
      });
    </script>
    <script type="text/javascript" src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
  


  

  

  


</body>
</html>
